# -*- coding: utf-8 -*-
"""
爬虫脚本：抓取英文语法网站上的指定模块链接
目标网址：https://letmeenglish.com/zh-hans/grammar-list/
"""

import requests
from bs4 import BeautifulSoup
import json
import sys

# 目标网址
URL = "https://letmeenglish.com/zh-hans/grammar-list/"

# 创建一个会话
session = requests.Session()

# 设置请求头，模拟浏览器访问
headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
}

def fetch_page(url):
    """获取网页内容"""
    try:
        response = session.get(url, headers=headers, timeout=10)
        response.raise_for_status()  # 如果响应状态码不是200，抛出HTTPError
        return response.text
    except requests.RequestException as e:
        print(f"请求失败: {e}")
        return None


def parse_links(html_content, target_section):
    """解析HTML内容，提取指定模块的链接"""
    soup = BeautifulSoup(html_content, 'html.parser')
    
    # 存储结果的字典
    links = {}
    
    # 查找所有h2标签（模块标题）
    h2_tags = soup.find_all('h2')
    
    for h2 in h2_tags:
        # 获取h2标签的文本内容
        h2_text = h2.get_text(strip=True)
        
        # 判断当前h2是否是目标模块
        if target_section in h2_text:
            # 检查是否有h3子分类
            has_h3_categories = False
            next_element = h2.find_next_sibling()
            while next_element and next_element.name != 'h2':
                if next_element.name == 'h3':
                    has_h3_categories = True
                    break
                next_element = next_element.find_next_sibling()
            
            if has_h3_categories:
                # 处理带子分类的模块（如英文时态）
                h3_tags = h2.find_all_next('h3')
                for h3 in h3_tags:
                    # 检查h3是否属于当前h2模块
                    if h3.find_previous('h2') != h2:
                        break
                        
                    h3_title = h3.get_text(strip=True)
                    ul_tag = h3.find_next('ul')
                    if ul_tag:
                        li_tags = ul_tag.find_all('li')
                        for li in li_tags:
                            a_tag = li.find('a')
                            if a_tag:
                                url = a_tag.get('href')
                                title = a_tag.get_text(strip=True)
                                full_url = f"https://letmeenglish.com{url}" if url.startswith('/') else url
                                
                                if target_section not in links:
                                    links[target_section] = []
                                links[target_section].append({
                                    "category": h3_title,
                                    "title": title,
                                    "url": full_url
                                })
            else:
                # 处理不带子分类的模块（如情态动词）
                next_sibling = h2.find_next_sibling()
                while next_sibling and next_sibling.name != 'h2':
                    if next_sibling.name == 'ul':
                        li_tags = next_sibling.find_all('li')
                        for li in li_tags:
                            a_tag = li.find('a')
                            if a_tag:
                                url = a_tag.get('href')
                                title = a_tag.get_text(strip=True)
                                full_url = f"https://letmeenglish.com{url}" if url.startswith('/') else url
                                
                                if target_section not in links:
                                    links[target_section] = []
                                links[target_section].append({
                                    "title": title,
                                    "url": full_url
                                })
                    next_sibling = next_sibling.find_next_sibling()
    
    return links

def main(target_section=None):
    """主函数"""
    # 检查命令行参数
    if target_section is None:
        if len(sys.argv) < 2:
            print("用法: python spider_grammar.py <模块名称>")
            print("可用模块: 现在时, 过去时, 将来时, 情态动词, 祈使句等, 条件句, 被动式, 动词不定式和动词-ing, 冠词、名词、代词和限定词, 关系从句、关系代词和副词, there and it, 助动词, 形容词和副词, 连接词和子句, 介词, 问句, 词序, 其他")
            return
        # 获取目标模块名称
        target_section = sys.argv[1]
    
    print(f"正在抓取 '{target_section}' 模块的内容...")
    html_content = fetch_page(URL)
    
    if html_content:
        print("正在解析网页内容...")
        links = parse_links(html_content, target_section)
        
        # 输出结果
        if target_section in links:
            print(f"\n=== {target_section} 相关链接 ===\n")
            # 处理带分类的数据
            if links[target_section] and 'category' in links[target_section][0]:
                # 按照分类组织输出
                categorized_links = {}
                for link in links[target_section]:
                    category = link['category']
                    if category not in categorized_links:
                        categorized_links[category] = []
                    categorized_links[category].append({
                        "title": link['title'],
                        "url": link['url']
                    })
                
                for category, link_list in categorized_links.items():
                    print(f"\n--- {category} ---\n")
                    for link in link_list:
                        print(f"- {link['title']}: {link['url']}")
            else:
                # 直接输出链接
                for link in links[target_section]:
                    print(f"- {link['title']}: {link['url']}")
        else:
            print(f"未找到 '{target_section}' 模块")
    else:
        print("无法获取网页内容")

if __name__ == '__main__':
    # 检查是否有命令行参数
    if len(sys.argv) >= 2:
        main(sys.argv[1])
    else:
        # main("英文时态")
        main('情态动词，祈使句等')